

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Preparation before Training &mdash; KoSpeech 0.0 documentation</title>
  

  
  
  
  

  
  <script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script type="text/javascript" src="../_static/jquery.js"></script>
        <script type="text/javascript" src="../_static/underscore.js"></script>
        <script type="text/javascript" src="../_static/doctools.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="Options" href="opts.html" />
    <link rel="prev" title="Intro" href="intro.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home"> KoSpeech
          

          
          </a>

          
            
            
              <div class="version">
                0.0
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">NOTES</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="intro.html">Intro</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Preparation before Training</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#data-structure"><em>Data Structure</em></a></li>
<li class="toctree-l2"><a class="reference internal" href="#base-function">Base Function</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#filenum-padding"><strong>filenum_padding()</strong></a></li>
<li class="toctree-l3"><a class="reference internal" href="#get-path"><strong>get_path()</strong></a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#data-preprocess"><em>Data-Preprocess</em></a><ul>
<li class="toctree-l3"><a class="reference internal" href="#bracket-filter"><strong>bracket_filter()</strong></a></li>
<li class="toctree-l3"><a class="reference internal" href="#special-filter"><strong>special_filter()</strong></a></li>
<li class="toctree-l3"><a class="reference internal" href="#sentence-filter"><strong>sentence_filter()</strong></a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#create-character-labels"><em>Create Character labels</em></a><ul>
<li class="toctree-l3"><a class="reference internal" href="#id1"><strong>Create Character labels</strong></a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#create-target-text"><em>Create target text</em></a><ul>
<li class="toctree-l3"><a class="reference internal" href="#sentence-to-target"><strong>sentence_to_target()</strong></a></li>
<li class="toctree-l3"><a class="reference internal" href="#target-to-sentence"><strong>target_to_sentence()</strong></a></li>
<li class="toctree-l3"><a class="reference internal" href="#create-new-label-text"><em>Create New label Text</em></a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#create-data-list"><em>Create data list</em></a></li>
<li class="toctree-l2"><a class="reference internal" href="#contacts"><strong>Contacts</strong></a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="opts.html">Options</a></li>
</ul>
<p class="caption"><span class="caption-text">ARCHITECTURE</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../Seq2seq.html">Seq2seq</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Transformer.html">Transformer</a></li>
</ul>
<p class="caption"><span class="caption-text">PACKAGE REFERENCE</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../Checkpoint.html">Checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Data.html">Data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Decode.html">Decode</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Evaluator.html">Evaluator</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Optim.html">Optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Trainer.html">Trainer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../Etc.html">Etc</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">KoSpeech</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
      <li>Preparation before Training</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/notes/Preparation.md.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="preparation-before-training">
<h1>Preparation before Training<a class="headerlink" href="#preparation-before-training" title="Permalink to this headline">¶</a></h1>
<p><a class="reference external" href="https://github.com/sooftware/KsponSpeech.preprocess">Preprocess GitHub Repository</a></p>
<p>본 글은 <strong>AI Hub</strong>에서 제공하는 ‘<strong>한국어 음성데이터</strong>’에 대해 학습 진행을 위한 과정을 기록한 글입니다.<br />AI Hub 음성 데이터는 다음 링크에서 신청 후 다운로드 하실 수 있습니다.</p>
<p>AI Hub 한국어 음성 데이터 : http://www.aihub.or.kr/aidata/105<br />GitHub Repository : https://github.com/sooftware/End-to-end-Speech-Recognition</p>
<div class="section" id="data-structure">
<h2><em>Data Structure</em><a class="headerlink" href="#data-structure" title="Permalink to this headline">¶</a></h2>
<p><img alt="data-structure" src="https://postfiles.pstatic.net/MjAyMDAyMjRfNjYg/MDAxNTgyNDc2NzExNDc4.xu4S2PfKcHun-I1dTODrzIQfcQTzjdjdiuVFnvtFh8Ug.kgTQvNWFfv_LoS0HjB91CMU_ochW8bSDybp7a51c3bog.PNG.sooftware/image.png?type=w773" /></p>
<p>데이터는 총 123GB로 크게 5개의 폴더로 이루어져 있고, 각 폴더 안에는 124개의 폴더가 있다. 그리고 그 폴더 안에는 1,000개씩의 PCM-TXT 파일로 구성되어 있다.  조용한 환경에서 2,000여명이 발성한 한국어 <strong>1,000시간</strong>의 데이터로 구성되어 있다. 총 파일의 개수는 <strong>622,545</strong>개의 PCM-TXT 파일로 구성되어 있다.</p>
<p>※ 작업의 편의를 위하여 아래부터 이루어지는 작업은 모든 파일을 하나의 폴더 안에 모아서 작업했습니다 ※</p>
<ul class="simple">
<li>KsponSpeech_FILENUM.pcm<br /><img alt="signal" src="https://postfiles.pstatic.net/MjAyMDAxMjJfMTYx/MDAxNTc5NjcyNzMyMTkz.Kw1WWrvvv9qLEf-pa0QYOcKYL3GOqXxahw_6sBsjqLgg.nkysalfeHToY9_FbVgxVcOM_Q5_RYlbpfFrAdFsdev4g.PNG.sooftware/audio-signal.png?type=w773" /></li>
<li>KsponSpeech_FILENUM.txt</li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="s2">&quot;b/ 아/ 모+ 몬 소리야 (70%)/(칠 십 퍼센트) 확률이라니 n/&quot;</span>  
</pre></div>
</div>
</div>
<div class="section" id="base-function">
<h2>Base Function<a class="headerlink" href="#base-function" title="Permalink to this headline">¶</a></h2>
<p>전처리를 위해 필요한 기본 함수들을 정의해보자.</p>
<div class="section" id="filenum-padding">
<h3><strong>filenum_padding()</strong><a class="headerlink" href="#filenum-padding" title="Permalink to this headline">¶</a></h3>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">filenum_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">):</span>
    <span class="k">if</span> <span class="n">filenum</span> <span class="o">&lt;</span> <span class="mi">10</span><span class="p">:</span> 
        <span class="k">return</span> <span class="s1">&#39;00000&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span>
    <span class="k">elif</span> <span class="n">filenum</span> <span class="o">&lt;</span> <span class="mi">100</span><span class="p">:</span> 
        <span class="k">return</span> <span class="s1">&#39;0000&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span>
    <span class="k">elif</span> <span class="n">filenum</span> <span class="o">&lt;</span> <span class="mi">1000</span><span class="p">:</span> 
        <span class="k">return</span> <span class="s1">&#39;000&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span>
    <span class="k">elif</span> <span class="n">filenum</span> <span class="o">&lt;</span> <span class="mi">10000</span><span class="p">:</span> 
        <span class="k">return</span> <span class="s1">&#39;00&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span>
    <span class="k">elif</span> <span class="n">filenum</span> <span class="o">&lt;</span> <span class="mi">100000</span><span class="p">:</span> 
        <span class="k">return</span> <span class="s1">&#39;0&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span> 
        <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span>
</pre></div>
</div>
<p>AI Hub 데이터셋에서 파일 번호는 ‘000001’, ‘002545’, ‘612543’ 와 같은 형식으로 이루어져 있다.<br />이러한 형식에 맞춰주기 위하여 파일 번호를 입력으로 받아 해당 포맷에 맞춰주는 함수를 미리 정의해둔다.</p>
</div>
<div class="section" id="get-path">
<h3><strong>get_path()</strong><a class="headerlink" href="#get-path" title="Permalink to this headline">¶</a></h3>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">get_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">fname</span><span class="p">,</span> <span class="n">filenum</span><span class="p">,</span> <span class="n">format</span><span class="p">):</span>
    <span class="k">return</span> <span class="n">path</span> <span class="o">+</span> <span class="n">fname</span> <span class="o">+</span> <span class="n">filenum</span> <span class="o">+</span> <span class="n">format</span>
</pre></div>
</div>
<p>텍스트 파일의 경로를 잡아주는 함수를 미리 정의해둔다.<br />Example )</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">BASE_PATH</span> <span class="o">=</span> <span class="s2">&quot;E:/한국어 음성데이터/KsponScript/&quot;</span>
<span class="n">FNAME</span> <span class="o">=</span> <span class="s1">&#39;KsponScript_&#39;</span>
<span class="n">filenum</span> <span class="o">=</span> <span class="mi">1348</span>
<span class="n">format</span> <span class="o">=</span> <span class="s1">&#39;.txt&#39;</span>

<span class="k">print</span><span class="p">(</span><span class="n">get_path</span><span class="p">(</span><span class="n">BASE_PATH</span><span class="p">,</span><span class="n">FNAME</span><span class="p">,</span><span class="n">filenum_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">),</span><span class="s2">&quot;.txt&quot;</span><span class="p">))</span>
</pre></div>
</div>
<p><strong>Output</strong></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="s1">&#39;E:/한국어 음성데이터/KsponSpeech/KsponScript_001348.txt&#39;</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="data-preprocess">
<h2><em>Data-Preprocess</em><a class="headerlink" href="#data-preprocess" title="Permalink to this headline">¶</a></h2>
<p>AI Hub에서 기본적으로 제공하는 음성에 대한 텍스트는 다음과 같다.<br />(철자전사) / (발음전사), 노이즈, 더듬는 소리 등 세밀하게 레이블링 되어 있다.<br />우리 팀은 <em>Sound-To-Text</em> 를 최대한 정확하게 하는 것이 목표였기에 다음과 같은 전처리 과정을 거쳤다.</p>
<ul class="simple">
<li>Raw Data</li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="s2">&quot;b/ 아/ 모+ 몬 소리야 (70%)/(칠 십 퍼센트) 확률이라니 n/&quot;</span>  
</pre></div>
</div>
<ul class="simple">
<li>b/, n/, / .. 등의 잡음 레이블 삭제</li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="s2">&quot;아/ 모+ 몬 소리야 (70%)/(칠 십 퍼센트) 확률이라니&quot;</span>
</pre></div>
</div>
<ul class="simple">
<li>제공된 (철자전사)/(발음전사) 중 발음전사 사용</li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="s2">&quot;아/ 모+ 몬 소리야 칠 십 퍼센트 확률이라니&quot;</span>
</pre></div>
</div>
<ul class="simple">
<li>간투어 표현 등을 위해 사용된 ‘/’, ‘*’, ‘+’ 등의 레이블 삭제</li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="s2">&quot;아 모 몬 소리야 칠 십 퍼센트 확률이라니&quot;</span>
</pre></div>
</div>
<p>다음은 위와 같은 전처리를 위해 사용한 코드이다.</p>
<div class="section" id="bracket-filter">
<h3><strong>bracket_filter()</strong><a class="headerlink" href="#bracket-filter" title="Permalink to this headline">¶</a></h3>
<p>(A) / (B) 일 때, B만을 가져와주는 함수이다.<br />(철자전사) / (발음전사) 중 발음전사를 선택하기 위해 정의했다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">test1</span> <span class="o">=</span> <span class="s2">&quot;o/ 근데 (70%)/(칠십 퍼센트)가 커 보이긴 하는데 (200)/(이백) 벌다 (140)/(백 사십) 벌면 빡셀걸? b/&quot;</span>
<span class="n">test2</span> <span class="o">=</span> <span class="s2">&quot;근데 (3학년)/(삼 학년) 때 까지는 국가장학금 바+ 받으면서 다녔던 건가?&quot;</span>

<span class="k">def</span> <span class="nf">bracket_filter</span><span class="p">(</span><span class="n">sentence</span><span class="p">):</span>
    <span class="n">new_sentence</span> <span class="o">=</span> <span class="nb">str</span><span class="p">()</span>
    <span class="n">flag</span> <span class="o">=</span> <span class="bp">False</span>
    
    <span class="k">for</span> <span class="n">ch</span> <span class="ow">in</span> <span class="n">sentence</span><span class="p">:</span>
        <span class="k">if</span> <span class="n">ch</span> <span class="o">==</span> <span class="s1">&#39;(&#39;</span> <span class="ow">and</span> <span class="n">flag</span> <span class="o">==</span> <span class="bp">False</span><span class="p">:</span> 
            <span class="n">flag</span> <span class="o">=</span> <span class="bp">True</span>
            <span class="k">continue</span>
        <span class="k">if</span> <span class="n">ch</span> <span class="o">==</span> <span class="s1">&#39;(&#39;</span> <span class="ow">and</span> <span class="n">flag</span> <span class="o">==</span> <span class="bp">True</span><span class="p">:</span>
            <span class="n">flag</span> <span class="o">=</span> <span class="bp">False</span>
            <span class="k">continue</span>
        <span class="k">if</span> <span class="n">ch</span> <span class="o">!=</span> <span class="s1">&#39;)&#39;</span> <span class="ow">and</span> <span class="n">flag</span> <span class="o">==</span> <span class="bp">False</span><span class="p">:</span>
            <span class="n">new_sentence</span> <span class="o">+=</span> <span class="n">ch</span>
    <span class="k">return</span> <span class="n">new_sentence</span>

<span class="k">print</span><span class="p">(</span><span class="n">bracket_filter</span><span class="p">(</span><span class="n">test1</span><span class="p">))</span>
<span class="k">print</span><span class="p">(</span><span class="n">bracket_filter</span><span class="p">(</span><span class="n">test2</span><span class="p">))</span>
</pre></div>
</div>
<p><strong>Output</strong></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="s1">&#39;o/ 근데 칠십 퍼센트가 커 보이긴 하는데 이백 벌다 백 사십 벌면 빡셀걸? b/&#39;</span>
<span class="s1">&#39;근데 삼 학년 때 까지는 국가장학금 바+ 받으면서 다녔던 건가?&#39;</span>
</pre></div>
</div>
</div>
<div class="section" id="special-filter">
<h3><strong>special_filter()</strong><a class="headerlink" href="#special-filter" title="Permalink to this headline">¶</a></h3>
<p>문자 단위로 특수 문자 및 노이즈 표기 필터링해주는 함수이다.<br />특수 문자를 아예 필터링 해버리면 문제가 되는 ‘#’, ‘%’와 같은 문자를 확인하고, 문제가 되는 특수 문자는 해당 발음으로 바꿔주었다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">re</span>

<span class="n">test1</span> <span class="o">=</span> <span class="s2">&quot;o/ 근데 칠십 퍼센트가 커 보이긴 하는데 이백 벌다 백 사십 벌면 빡셀걸? b/&quot;</span>
<span class="n">test2</span> <span class="o">=</span> <span class="s2">&quot;c# 배워봤어?&quot;</span>

<span class="k">def</span> <span class="nf">special_filter</span><span class="p">(</span><span class="n">sentence</span><span class="p">):</span>
    <span class="n">SENTENCE_MARK</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;?&#39;</span><span class="p">,</span> <span class="s1">&#39;!&#39;</span><span class="p">]</span>
    <span class="n">NOISE</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;o&#39;</span><span class="p">,</span> <span class="s1">&#39;n&#39;</span><span class="p">,</span> <span class="s1">&#39;u&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">]</span>
    <span class="n">EXCEPT</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="s1">&#39;+&#39;</span><span class="p">,</span> <span class="s1">&#39;*&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;@&#39;</span><span class="p">,</span> <span class="s1">&#39;$&#39;</span><span class="p">,</span> <span class="s1">&#39;^&#39;</span><span class="p">,</span> <span class="s1">&#39;&amp;&#39;</span><span class="p">,</span> <span class="s1">&#39;[&#39;</span><span class="p">,</span> <span class="s1">&#39;]&#39;</span><span class="p">,</span> <span class="s1">&#39;=&#39;</span><span class="p">,</span> <span class="s1">&#39;:&#39;</span><span class="p">,</span> <span class="s1">&#39;;&#39;</span><span class="p">,</span> <span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;,&#39;</span><span class="p">]</span>
    
    <span class="n">new_sentence</span> <span class="o">=</span> <span class="nb">str</span><span class="p">()</span>
    <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">ch</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">sentence</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">ch</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">SENTENCE_MARK</span><span class="p">:</span>
            <span class="c1"># o/, n/ 등 처리</span>
            <span class="k">if</span> <span class="n">idx</span> <span class="o">+</span> <span class="mi">1</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">sentence</span><span class="p">)</span> <span class="ow">and</span> <span class="n">ch</span> <span class="ow">in</span> <span class="n">NOISE</span> <span class="ow">and</span> <span class="n">sentence</span><span class="p">[</span><span class="n">idx</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;/&#39;</span><span class="p">:</span> 
                <span class="k">continue</span> 

        <span class="k">if</span> <span class="n">ch</span> <span class="o">==</span> <span class="s1">&#39;#&#39;</span><span class="p">:</span> 
            <span class="n">new_sentence</span> <span class="o">+=</span> <span class="s1">&#39;샾&#39;</span>

        <span class="k">elif</span> <span class="n">ch</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">EXCEPT</span><span class="p">:</span> 
            <span class="n">new_sentence</span> <span class="o">+=</span> <span class="n">ch</span>

    <span class="n">pattern</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;\s\s+&#39;</span><span class="p">)</span>
    <span class="n">new_sentence</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="n">pattern</span><span class="p">,</span> <span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="n">new_sentence</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
    <span class="k">return</span> <span class="n">new_sentence</span>

<span class="k">print</span><span class="p">(</span><span class="n">special_filter</span><span class="p">(</span><span class="n">test1</span><span class="p">))</span>
<span class="k">print</span><span class="p">(</span><span class="n">special_filter</span><span class="p">(</span><span class="n">test2</span><span class="p">))</span>
</pre></div>
</div>
<p><strong>Output</strong></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="s1">&#39;근데 칠십 퍼센트가 커 보이긴 하는데 이백 벌다 백 사십 벌면 빡셀걸?&#39;</span>
<span class="s1">&#39;c샾 배워봤어?&#39;</span>
</pre></div>
</div>
<p><strong>&lt; . &gt;, &lt; , &gt;</strong> 같은 문장 부호는 음성인식 태스크에서 중요하지 않을 뿐더러 음성신호만으로 예측하기 어렵다고 생각하여 제외하였다. <strong>&lt; ? &gt;, &lt; ! &gt;</strong> 는 음성 신호로부터 예측 가능하고 중요한 문장 부호라고 생각하여 필터링하지 않았다.</p>
</div>
<div class="section" id="sentence-filter">
<h3><strong>sentence_filter()</strong><a class="headerlink" href="#sentence-filter" title="Permalink to this headline">¶</a></h3>
<p>위에서 정의한 2 함수를 이용해서 문장을 필터링해주는 함수</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">test</span> <span class="o">=</span> <span class="s2">&quot;o/ 근데 (70%)/(칠십 퍼센트)가 커 보이긴 하는데 (200)/(이백) 벌다 (140)/(백 사십) 벌면 빡셀걸? b/&quot;</span>

<span class="k">def</span> <span class="nf">sentence_filter</span><span class="p">(</span><span class="n">raw_sentence</span><span class="p">):</span>
    <span class="k">return</span> <span class="n">special_filter</span><span class="p">(</span><span class="n">bracket_filter</span><span class="p">(</span><span class="n">raw_sentence</span><span class="p">))</span>

<span class="k">print</span><span class="p">(</span><span class="n">sentence_filter</span><span class="p">(</span><span class="n">test</span><span class="p">))</span>
</pre></div>
</div>
<p><strong>Output</strong></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="s1">&#39;근데 칠십 퍼센트가 커 보이긴 하는데 이백 벌다 백 사십 벌면 빡셀걸?&#39;</span>
</pre></div>
</div>
<p>위의 과정을 끝내면 <strong>&lt; % &gt;</strong> 특수문자가 남게되는데, 해당 특수문자는 <strong>‘프로’, ‘퍼센트’</strong> 두 가지 발음이 가능하므로, 직접 확인한 결과 총 8개의 파일에서 등장했고, 4개의 **’프로’**와 4개의 <strong>‘퍼센트’</strong> 이루어지는 것을 확인하고 수작업으로 변환했다.</p>
</div>
</div>
<div class="section" id="create-character-labels">
<h2><em>Create Character labels</em><a class="headerlink" href="#create-character-labels" title="Permalink to this headline">¶</a></h2>
<p>위와 같이 AI Hub에서 제공되는 텍스트는 일정한 포맷의 ‘<strong>한글</strong>’로 구성되어 있다. 위의 한글 텍스트로는 학습을 시킬수가 없으므로, 컴퓨터가 이해할 수 있도록 ‘<strong>숫자</strong>’로 바꾸어 줘야 한다.</p>
<p>그러기 위해서 먼저, 데이터셋이 어떠한 문자들로 이루어져 있는지를 파악해야한다.</p>
<p>그럼 데이터셋에서 등장하는 모든 문자를 확인을 해보자.</p>
<div class="section" id="id1">
<h3><strong>Create Character labels</strong><a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>
<span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">trange</span> <span class="c1"># display progress</span>

<span class="n">BASE_PATH</span> <span class="o">=</span> <span class="s2">&quot;E:/한국어 음성데이터/KsponSpeech/&quot;</span>
<span class="n">FNAME</span> <span class="o">=</span> <span class="s1">&#39;KsponSpeech_&#39;</span>
<span class="n">TOTAL_NUM</span> <span class="o">=</span> <span class="mi">622545</span>
<span class="n">label_list</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">label_freq</span> <span class="o">=</span> <span class="p">[]</span>

<span class="k">print</span><span class="p">(</span><span class="s1">&#39;started...&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">filenum</span> <span class="ow">in</span> <span class="n">trange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">TOTAL_NUM</span><span class="p">):</span>
    <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">get_path</span><span class="p">(</span><span class="n">BASE_PATH</span><span class="p">,</span><span class="n">FNAME</span><span class="p">,</span><span class="n">filenum_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">),</span><span class="s2">&quot;.txt&quot;</span><span class="p">))</span>
    <span class="n">sentence</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
    <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
    <span class="k">for</span> <span class="n">ch</span> <span class="ow">in</span> <span class="n">sentence</span><span class="p">:</span>
        <span class="k">if</span> <span class="n">ch</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">label_list</span><span class="p">:</span>
            <span class="n">label_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ch</span><span class="p">)</span>
            <span class="n">label_freq</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">label_freq</span><span class="p">[</span><span class="n">label_list</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">ch</span><span class="p">)]</span> <span class="o">+=</span> <span class="mi">1</span>

<span class="c1"># sort together Using zip</span>
<span class="n">label_freq</span><span class="p">,</span> <span class="n">label_list</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="o">*</span><span class="nb">sorted</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">label_freq</span><span class="p">,</span> <span class="n">label_list</span><span class="p">),</span> <span class="n">reverse</span><span class="o">=</span><span class="bp">True</span><span class="p">))</span>
<span class="n">label</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;id&#39;</span><span class="p">:[],</span> <span class="s1">&#39;char&#39;</span><span class="p">:[],</span> <span class="s1">&#39;freq&#39;</span> <span class="p">:[]}</span>
<span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="p">(</span><span class="n">ch</span><span class="p">,</span> <span class="n">freq</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">label_list</span><span class="p">,</span> <span class="n">label_freq</span><span class="p">)):</span>
    <span class="n">label</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span>
    <span class="n">label</span><span class="p">[</span><span class="s1">&#39;char&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ch</span><span class="p">)</span>
    <span class="n">label</span><span class="p">[</span><span class="s1">&#39;freq&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">freq</span><span class="p">)</span>

<span class="sd">&quot;&quot;&quot; dictionary to csv &quot;&quot;&quot;</span>
<span class="n">label_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">label</span><span class="p">)</span>
<span class="n">label_df</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s2">&quot;aihub_labels.csv&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
</pre></div>
</div>
<p>위의 코드를 실행시켜서 정상적으로 종료됐다면 다음과 같은 파일이 생길 것이다.</p>
<p>|id|meaning|<br />|:–:|:–:|<br />|&lt;s&gt;|Start Of Sentence|<br />|&lt;/s&gt;|End Of Sentence|<br />|_|PAD|</p>
<p>( 위의 문자들은 수동으로 추가 )</p>
<p>|id|char|freq|<br />|:–:|:—-:|:—-:|<br />|0| |5774462|<br />|1|.|640924|<br />|2|그|556373|<br />|3|이|509291|<br />|.|.|.|<br />|.|.|.|<br />|2334|&lt;s&gt;|0|<br />|2335|&lt;/s&gt;|0|<br />|2336|_|0|</p>
<p>수동으로 추가해 준 3개의 레이블을 포함하여 총 <strong>2,337</strong>개의 문자 레이블이 완성되었다.<br />우리 팀은 위의 레이블 리스트 중 1번씩 등장한 문자에 주목했다.</p>
<p>‘갗’, ‘괞’, ‘긃’ 등의 생소한 문자가 약 300개 정도를 차지했는데   우리 팀은 이러한 레이블은 노이즈가 될 것이라고 생각했고, 이에 대한 처리를 고민했다. 1번씩 등장한 파일들을 확인해서 하나하나 수작업으로 레이블을 바꿔주려 했지만, 실제로 음성 파일을 들어보게 되면 바꿔주기가 상당히 애매했다.</p>
<p>|id|char|freq|<br />|:–:|:—-:|:—-:|<br />|0| |5774462|<br />|1|.|640924|<br />|2|그|556373|<br />|3|이|509291|<br />|.|.|.|<br />|.|.|.|<br />|2037|&lt;s&gt;|0|<br />|2038|&lt;/s&gt;|0|<br />|2039|_|0|</p>
<p>그래서 우리는 이렇게 1번씩 등장한 문자가 포함된 파일은 <strong>테스트 데이터</strong>로 사용하고, 2번 이상 등장한 문자들만 있는 파일들로만 <strong>트레이닝 데이터</strong>를 구성했다. 이렇게 1번씩 등장한 300여개의 문자들이 포함된 파일들을 제외를 해서, 위의 표처럼 <strong>2,040</strong>개의 문자로만 트레이닝을 시킬 수 있었다.</p>
<ul class="simple">
<li>train_labels.csv : 1번씩 등장한 문자를 제외한 <strong>2,040</strong>개의 문자 레이블</li>
<li>test_labels.csv : 데이터셋에서 등장한 <strong>2,337</strong>개의 문자 레이블<br />링크 : https://github.com/sh951011/Korean-Speech-Recognition/tree/master/data/label</li>
</ul>
</div>
</div>
<div class="section" id="create-target-text">
<h2><em>Create target text</em><a class="headerlink" href="#create-target-text" title="Permalink to this headline">¶</a></h2>
<p>이제 위에서 만든 <strong>Character label</strong>을 이용해서 한글로 이루어진 텍스트를 숫자로 바꿔보자.<br />먼저 만들어둔 문자 레이블 파일을 이용하여 문자를 id로, id를 문자로 만드는 딕셔너리 변수를 만든다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>

<span class="k">def</span> <span class="nf">load_label</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
    <span class="n">char2id</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
    <span class="n">id2char</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
    <span class="n">ch_labels</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;cp949&quot;</span><span class="p">)</span>
    <span class="n">id_list</span> <span class="o">=</span> <span class="n">ch_labels</span><span class="p">[</span><span class="s2">&quot;id&quot;</span><span class="p">]</span>
    <span class="n">char_list</span> <span class="o">=</span> <span class="n">ch_labels</span><span class="p">[</span><span class="s2">&quot;char&quot;</span><span class="p">]</span>
    <span class="n">freq_list</span> <span class="o">=</span> <span class="n">ch_labels</span><span class="p">[</span><span class="s2">&quot;freq&quot;</span><span class="p">]</span>
    
    <span class="k">for</span> <span class="p">(</span><span class="nb">id</span><span class="p">,</span> <span class="n">char</span><span class="p">,</span> <span class="n">freq</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">id_list</span><span class="p">,</span> <span class="n">char_list</span><span class="p">,</span> <span class="n">freq_list</span><span class="p">):</span>
        <span class="n">char2id</span><span class="p">[</span><span class="n">char</span><span class="p">]</span> <span class="o">=</span> <span class="nb">id</span>
        <span class="n">id2char</span><span class="p">[</span><span class="nb">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">char</span>
    <span class="k">return</span> <span class="n">char2id</span><span class="p">,</span> <span class="n">id2char</span>
</pre></div>
</div>
<p>이제 <strong>char2id</strong> 변수를 이용하여 한글 텍스트 문장을 숫자로 바꿔보자.</p>
<div class="section" id="sentence-to-target">
<h3><strong>sentence_to_target()</strong><a class="headerlink" href="#sentence-to-target" title="Permalink to this headline">¶</a></h3>
<p>한글 텍스트를 숫자 레이블로 변환해주는 함수</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">test</span> <span class="o">=</span> <span class="s1">&#39;근데 칠십 퍼센트가 커 보이긴 하는데 이백 벌다 백 사십 벌면 빡셀걸?&#39;</span>

<span class="k">def</span> <span class="nf">sentence_to_target</span><span class="p">(</span><span class="n">sentence</span><span class="p">,</span> <span class="n">char2id</span><span class="p">):</span>
    <span class="n">target</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
    <span class="k">for</span> <span class="n">ch</span> <span class="ow">in</span> <span class="n">sentence</span><span class="p">:</span>
        <span class="n">target</span> <span class="o">+=</span> <span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">char2id</span><span class="p">[</span><span class="n">ch</span><span class="p">])</span> <span class="o">+</span> <span class="s1">&#39; &#39;</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">target</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>

<span class="k">print</span><span class="p">(</span><span class="n">sentence_to_target</span><span class="p">(</span><span class="n">test</span><span class="p">,</span> <span class="n">char2id</span><span class="p">))</span>
</pre></div>
</div>
<p><strong>Output</strong></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="s1">&#39;35 11 0 318 119 0 489 551 156 6 0 379 0 42 3 144 0 14 4 11 0 3 248 0 355 15 0 248 0 34 119 0 355 24 0 588 785 104 12&#39;</span>
</pre></div>
</div>
<p>텍스트를 미리 만들어둔 문자 레이블을 이용하여 숫자로 바꿔주었다.<br />숫자끼리는 띄어쓰기로 구분해주었다.<br />문장 마지막에 붙는 띄어쓰기는 의미가 없어서 target[:-1]로 제거해준 뒤 반환했다.</p>
<p>잘 변환되었는지 확인하기 위해 이 숫자 레이블을 다시 한글 텍스트로 변환해보자.</p>
</div>
<div class="section" id="target-to-sentence">
<h3><strong>target_to_sentence()</strong><a class="headerlink" href="#target-to-sentence" title="Permalink to this headline">¶</a></h3>
<p>숫자 레이블을 한글 텍스트로 변환해주는 함수</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">test</span> <span class="o">=</span> <span class="s1">&#39;35 11 0 318 119 0 489 551 156 6 0 379 0 42 3 144 0 14 4 11 0 3 248 0 355 15 0 248 0 34 119 0 355 24 0 588 785 104 12&#39;</span>

<span class="k">def</span> <span class="nf">target_to_sentence</span><span class="p">(</span><span class="n">target</span><span class="p">,</span> <span class="n">id2char</span><span class="p">):</span>
    <span class="n">sentence</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
    <span class="n">targets</span> <span class="o">=</span> <span class="n">target</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>

    <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span>
        <span class="n">sentence</span> <span class="o">+=</span> <span class="n">id2char</span><span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="p">)]</span>
    <span class="k">return</span> <span class="n">sentence</span>

<span class="k">print</span><span class="p">(</span><span class="n">target_to_sentence</span><span class="p">(</span><span class="n">test</span><span class="p">,</span> <span class="n">id2char</span><span class="p">))</span>
</pre></div>
</div>
<p><strong>Output</strong></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="s1">&#39;근데 칠십 퍼센트가 커 보이긴 하는데 이백 벌다 백 사십 벌면 빡셀걸?&#39;</span>
</pre></div>
</div>
<p>기존 한글 텍스트로 정상적으로 변환된 것을 확인했다.<br />이로써 한글 텍스트에서 숫자레이블로, 숫자레이블에서 한글 텍스트로의 변환을 자유자재로 할 수 있게 되었다.<br />이제 이 함수들을 이용해서 전체 데이터셋에 대하여 변환해주기만 하면 된다.</p>
</div>
<div class="section" id="create-new-label-text">
<h3><em>Create New label Text</em><a class="headerlink" href="#create-new-label-text" title="Permalink to this headline">¶</a></h3>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>
<span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">trange</span> <span class="c1"># display progress</span>

<span class="n">BASE_PATH</span> <span class="o">=</span> <span class="s2">&quot;E:/한국어 음성데이터/KsponSpeech/&quot;</span>
<span class="n">FNAME</span> <span class="o">=</span> <span class="s1">&#39;KsponSpeech_&#39;</span>
<span class="n">NEW_FNAME</span> <span class="o">=</span> <span class="s1">&#39;KsponScript_&#39;</span>
<span class="n">TOTAL_NUM</span> <span class="o">=</span> <span class="mi">622545</span>
<span class="n">char2id</span><span class="p">,</span> <span class="n">id2char</span> <span class="o">=</span> <span class="n">load_label</span><span class="p">(</span><span class="s2">&quot;test_labels.csv&quot;</span><span class="p">)</span>

<span class="k">print</span><span class="p">(</span><span class="s1">&#39;started...&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">filenum</span> <span class="ow">in</span> <span class="n">trange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">TOTAL_NUM</span><span class="p">):</span>
    <span class="n">sentence</span><span class="p">,</span> <span class="n">target</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span> <span class="bp">None</span>

    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">get_path</span><span class="p">(</span><span class="n">BASE_PATH</span><span class="p">,</span><span class="n">FNAME</span><span class="p">,</span><span class="n">filenum_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">),</span><span class="s2">&quot;.txt&quot;</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
        <span class="n">sentence</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
    
    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">get_path</span><span class="p">(</span><span class="n">BASE_PATH</span><span class="p">,</span> <span class="n">NEW_FNAME</span><span class="p">,</span> <span class="n">filenum_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">),</span><span class="s2">&quot;.txt&quot;</span><span class="p">),</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
        <span class="n">target</span> <span class="o">=</span> <span class="n">sentence_to_target</span><span class="p">(</span><span class="n">sentence</span><span class="p">,</span> <span class="n">char2id</span><span class="p">)</span>
        <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">target</span><span class="p">)</span>
</pre></div>
</div>
<p>위 까지의 과정을 통해서 기존 Raw 데이터셋에서 우리가 원하는 데이터셋의 형태로의 변환 및 숫자 레이블로의 변환까지 마쳤다.</p>
</div>
</div>
<div class="section" id="create-data-list">
<h2><em>Create data list</em><a class="headerlink" href="#create-data-list" title="Permalink to this headline">¶</a></h2>
<p>이제 학습을 진행하기 위해 위와 같은 트레이닝 / 테스트 데이터 리스트를 만들어보자.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>

<span class="n">TOTAL_NUM</span> <span class="o">=</span> <span class="mi">622545</span>
<span class="n">TRAIN_NUM</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span> <span class="mi">622545</span> <span class="o">*</span> <span class="mf">0.98</span> <span class="p">)</span>
<span class="n">TEST_NUM</span> <span class="o">=</span> <span class="n">TOTAL_NUM</span> <span class="o">-</span> <span class="n">TRAIN_NUM</span>

<span class="n">train_data_list</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;audio&#39;</span><span class="p">:[],</span> <span class="s1">&#39;label&#39;</span><span class="p">:[]}</span>
<span class="n">test_data_list</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;audio&#39;</span><span class="p">:[],</span> <span class="s1">&#39;label&#39;</span><span class="p">:[]}</span>

<span class="n">aihub_labels</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s2">&quot;test_labels.csv&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;cp949&#39;</span><span class="p">)</span>
<span class="n">rare_labels</span> <span class="o">=</span> <span class="n">aihub_labels</span><span class="p">[</span><span class="s1">&#39;char&#39;</span><span class="p">][</span><span class="mi">2037</span><span class="p">:]</span>
</pre></div>
</div>
<p>먼저 우리는 총 1,000시간 중 980시간을 훈련데이터, 20시간을 테스트데이터로 사용할 것이기 때문에 총 파일 개수의 98%에 해당하는 숫자를 TRAIN_NUM으로 계산해주고 나머지 숫자를 TEST_NUM으로 정의해주었다.<br />미리 만들어 둔 test_labels.csv 파일을 불러오고, 1번만 언급된 문자들은 rare_labels에 저장해준다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">trange</span>

<span class="n">fname</span> <span class="o">=</span> <span class="s1">&#39;KsponSpeech_&#39;</span>
<span class="n">target_fname</span> <span class="o">=</span> <span class="s1">&#39;KsponScript_&#39;</span>

<span class="n">audio_paths</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">target_paths</span> <span class="o">=</span> <span class="p">[]</span>

<span class="k">for</span> <span class="n">filenum</span> <span class="ow">in</span> <span class="n">trange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">TOTAL_NUM</span><span class="p">):</span>
    <span class="n">audio_paths</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fname</span> <span class="o">+</span> <span class="n">file_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;.pcm&quot;</span><span class="p">)</span>
    <span class="n">target_paths</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">target_fname</span> <span class="o">+</span> <span class="n">file_padding</span><span class="p">(</span><span class="n">filenum</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;.txt&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>다음으로 오디오 파일과 타겟 텍스트 파일 경로를 리스트에 저장한다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">random</span>

<span class="n">data_paths</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">audio_paths</span><span class="p">,</span> <span class="n">target_paths</span><span class="p">))</span>
<span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">data_paths</span><span class="p">)</span>
<span class="n">audio_paths</span><span class="p">,</span> <span class="n">target_paths</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="o">*</span><span class="n">data_paths</span><span class="p">)</span>
</pre></div>
</div>
<p>1번부터 622545번까지 순서대로 넣었으므로, 순서를 섞어준다.<br />이때, audio_paths와 target_paths를 같은 순서로 섞어주기 위해 zip()을 이용하여 섞어준다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">trange</span>

<span class="n">path</span> <span class="o">=</span> <span class="s2">&quot;G:/한국어 음성데이터/KsponSpeech/&quot;</span>
<span class="n">train_full</span> <span class="o">=</span> <span class="bp">False</span>
<span class="n">train_dict</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;audio&#39;</span><span class="p">:</span> <span class="p">[],</span>
    <span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="p">[]</span>
<span class="p">}</span>
<span class="n">test_dict</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;audio&#39;</span><span class="p">:</span> <span class="p">[],</span>
    <span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="p">[]</span>
<span class="p">}</span>

<span class="k">print</span><span class="p">(</span><span class="s1">&#39;started...&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="n">trange</span><span class="p">(</span><span class="n">length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">audio_paths</span><span class="p">)):</span>
    <span class="n">audio</span> <span class="o">=</span> <span class="n">audio_paths</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
    <span class="n">target</span> <span class="o">=</span> <span class="n">target_paths</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>

    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_dict</span><span class="p">[</span><span class="s1">&#39;audio&#39;</span><span class="p">])</span> <span class="o">==</span> <span class="n">TRAIN_NUM</span><span class="p">:</span>
        <span class="n">train_full</span> <span class="o">=</span> <span class="bp">True</span>

    <span class="k">if</span> <span class="n">train_full</span><span class="p">:</span>
        <span class="n">test_dict</span><span class="p">[</span><span class="s1">&#39;audio&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">audio</span><span class="p">)</span>
        <span class="n">test_dict</span><span class="p">[</span><span class="s1">&#39;label&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">label</span><span class="p">)</span>
        
    <span class="k">else</span><span class="p">:</span>
        <span class="n">rare_in</span> <span class="o">=</span> <span class="bp">False</span>
        <span class="n">sentence</span> <span class="o">=</span> <span class="bp">None</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">((</span><span class="n">path</span><span class="o">+</span><span class="n">audio</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">+</span><span class="s2">&quot;.txt&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> 
            <span class="n">sentence</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
            
        <span class="k">for</span> <span class="n">rare</span> <span class="ow">in</span> <span class="n">rare_labels</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">rare</span> <span class="ow">in</span> <span class="n">sentence</span><span class="p">:</span>
                <span class="n">rare_in</span> <span class="o">=</span> <span class="bp">True</span>
                <span class="k">break</span>
        <span class="k">if</span> <span class="n">rare_in</span><span class="p">:</span>
            <span class="n">test_dict</span><span class="p">[</span><span class="s1">&#39;audio&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">audio</span><span class="p">)</span>
            <span class="n">test_dict</span><span class="p">[</span><span class="s1">&#39;label&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">label</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">train_dict</span><span class="p">[</span><span class="s1">&#39;audio&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">audio</span><span class="p">)</span>
            <span class="n">train_dict</span><span class="p">[</span><span class="s1">&#39;label&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">label</span><span class="p">)</span>
            
<span class="k">print</span><span class="p">(</span><span class="s2">&quot;Ended!!!&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>랜덤하게 섞여진 audio_paths와 target_paths를 이용하여 train_dict와 test_dict를 만든다.<br />이때 rare_labels에 있는 문자가 포함된 파일은 test_dict로 넣어준다.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">test_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">test_dict</span><span class="p">)</span>
<span class="n">train_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">train_dict</span><span class="p">)</span>

<span class="n">test_df</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s2">&quot;test_list.csv&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;cp949&#39;</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
<span class="n">train_df</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s2">&quot;train_list.csv&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;cp949&#39;</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
</pre></div>
</div>
<p>train_dict와 test_dict를 DataFrame 형태로 변환 후에 to_csv 메서드로 csv 파일로 만들어준다.</p>
<ul class="simple">
<li>train_list.csv<br />학습용 데이터 리스트 - <strong>980h</strong></li>
</ul>
<p>| pcm-filename| txt-filename|<br />| :——————-| :————————–|<br />| KaiSpeech_078903.pcm | KaiSpeech_label_078903.txt  |<br />| KaiSpeech_449461.pcm | KaiSpeech_label_449461.txt  |<br />| KaiSpeech_178531.pcm | KaiSpeech_label_178531.txt  |<br />| KaiSpeech_374874.pcm | KaiSpeech_label_374874.txt  |<br />| KaiSpeech_039018.pcm | KaiSpeech_label_039018.txt  |</p>
<ul class="simple">
<li>test_list.csv<br />테스트용 데이터 리스트  - <strong>20h</strong></li>
</ul>
<p>| pcm-filaname| txt-filename|<br />| :——————-| :————————–|<br />| KaiSpeech_126887.pcm | KaiSpeech_label_126887.txt  |<br />| KaiSpeech_067340.pcm | KaiSpeech_label_067340.txt  |<br />| KaiSpeech_350293.pcm | KaiSpeech_label_350293.txt  |<br />| KaiSpeech_212197.pcm | KaiSpeech_label_212197.txt  |<br />| KaiSpeech_489840.pcm | KaiSpeech_label_489840.txt  |</p>
<p>data list 링크 : https://github.com/sh951011/Korean-Speech-Recognition/tree/master/data/data_list</p>
</div>
<div class="section" id="contacts">
<h2><strong>Contacts</strong><a class="headerlink" href="#contacts" title="Permalink to this headline">¶</a></h2>
<p>해당 작업에 대하여 에러, 문의사항 모두 환영합니다.<br />sh951011&#64;gmail.com 으로 연락주시면 감사하겠습니다.</p>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="opts.html" class="btn btn-neutral float-right" title="Options" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="intro.html" class="btn btn-neutral float-left" title="Intro" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2020, Soohwan Kim

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>